# -*- coding: UTF-8 -*- 
import urllib
import urllib2
import cookielib
from urllib2 import urlopen, Request, HTTPError
import threading
import re
import sys
import sqlite3
from worker import WorkerManager

headers = {
    'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)',
    'Accept': '*/*',
    'Accept-Language': 'zh-cn',
    'Accept-Charset': 'gb2312,utf-8;q=0.7,*;q=0.7',
}

class Spider(object):
    def __init__(self, configs, max_work_thread=2, max_rejection_num=100):
        self.configs = configs
        self.max_work_thread = max_work_thread
        self.max_rejection_num = max_rejection_num
    
    def start(self):        
        #分配任务给不同线程(线程池)
        wm = WorkerManager(self.max_work_thread)
        for config in self.configs:
            wm.add_job(self.login, config)
        wm.start()
        wm.wait_for_complete()
    
    def login(self, config):
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookielib.CookieJar()))
        urllib2.install_opener(opener)
        
        if 'init_fun' in config and callable(config['init_fun']):
            config['init_fun'](config)
        
        login_url = config['url'] + 'logging.php?action=login'
        
        #获得登录页面中的formhash
        req = Request(url=login_url, headers=headers)
        login_page_doc = urlopen(req).read()
        p = re.compile(r'<input type="hidden" name="formhash" value="(.*?)" \/>', re.MULTILINE)
        formhash = p.search(login_page_doc).group(1)
        #print formhash
        
        login_data = urllib.urlencode({
            'questionid': 0,
            'answer': '',
            'styleid': '',
            'cookietime': '2592000',
            'formhash': formhash,
            'loginfield': 'username',
            'loginmode': '',
            'loginsubmit': True,
            'username': config['username'],
            'password': config['password'],
            'referer': config['url'] + 'index.php'
        })
        
        req = Request(login_url, login_data, headers)
        conn = urlopen(req)
        
        p = re.compile(u'欢迎您回来，', re.MULTILINE)
        if p.search(conn.read().decode(config['encoding'])):
            print config['url'].ljust(30) + u'自动登录成功，开始抓取Email'
            self.get_content(config)
        else:
            print config['url'].ljust(30) + u'登录失败，请检查配置是否正确'
    
    def get_content(self, config):
        try:
            #创建数据库连接
            conn = sqlite3.connect('./email.s3db', isolation_level=None)
            
            #如果没有email表，则创建它
            cursor = conn.cursor()
            cursor.execute("CREATE TABLE IF NOT EXISTS [email] (\
                            [id] INTEGER  NOT NULL PRIMARY KEY,\
                            [email] VARCHAR(128)  NULL\
                            )")
            conn.commit()
            cursor.close()
            
            i = config['start_url_index']
            not_found_count = 0
            while True:
                #print config['url'] + 'space-uid-%d.html' % i
                
                req = Request(url=config['url'] + 'space-uid-%d.html' % i, headers=headers)
                doc = urlopen(req).read().decode(config['encoding'], 'ignore')
                #print doc
                
                p = re.compile(u'指定的用户不存在或已被删除', re.MULTILINE)
                #print doc
                is_not_found = p.search(doc)
                if is_not_found:
                    #如果连续max_rejection_num次页面没找到，则认为数据扒完了
                    if not_found_count < self.max_rejection_num:
                        not_found_count += 1
                    else:
                        print config['url'].ljust(30) + u'Email抓取完毕'
                        return 
                else:
                    #如果页面读取成功，则计数器清零
                    not_found_count = 0
                    
                    try:
                        p = re.compile(r'(([A-Za-z0-9]+_+)|([A-Za-z0-9]+\-+)|([A-Za-z0-9]+\.+)|([A-Za-z0-9]+\++))*[A-Za-z0-9]+&#64;((\w+\-+)|(\w+\.))*\w{1,63}\&#46;[a-zA-Z]{2,6}', re.MULTILINE)
                        m = p.search(doc)
                        if m:
                            email = m.group().replace('&#64;', '@').replace('&#46;', '.')
                            print email.ljust(30) + 'thread_name:' + threading.currentThread().getName().ljust(15) + config['url'] + 'space-uid-%d.html' % i 
                            
                            #存入到数据库中
                            c = conn.cursor()
                            c.execute("INSERT INTO email(email) values('%s')" % email)
                            conn.commit()
                            c.close()
                    except:
                        print sys.exc_info()
                        continue
                    
                i += 1
        except :
            print sys.exc_info()
            return 